# Define your item pipelines here
#
# Don't forget to add your pipeline to the ITEM_PIPELINES setting
# See: https://docs.scrapy.org/en/latest/topics/item-pipeline.html


# useful for handling different item types with a single interface
from itemadapter import ItemAdapter
import json
import os
import datetime
import sys
import importlib.util
import importlib.machinery


class FinanceScraperPipeline:
    def __init__(self):
        self.file_handlers = {}
        self.data_dir = 'finance_data'
        os.makedirs(self.data_dir, exist_ok=True)
        
        # Create directories for different data types
        for data_type in ['revenue', 'expenditure', 'tax', 'fund']:
            type_dir = os.path.join(self.data_dir, data_type)
            os.makedirs(type_dir, exist_ok=True)
    
    def open_spider(self, spider):
        # Initialize JSON files for different data types
        current_date = datetime.datetime.now().strftime('%Y%m%d')
        
        for data_type in ['revenue', 'expenditure', 'tax', 'fund']:
            file_path = os.path.join(self.data_dir, data_type, f'{data_type}_{current_date}.json')
            self.file_handlers[data_type] = open(file_path, 'w', encoding='utf-8')
            # Initialize with empty array
            self.file_handlers[data_type].write('[]')
            self.file_handlers[data_type].seek(0)
    
    def close_spider(self, spider):
        # Close all file handlers
        for handler in self.file_handlers.values():
            handler.close()
            
        # Update visualization
        self.update_visualization()
    
    def process_item(self, item, spider):
        adapter = ItemAdapter(item)
        data_type = adapter.get('data_type')
        
        if data_type in self.file_handlers:
            # Read existing data
            self.file_handlers[data_type].seek(0)
            data = json.loads(self.file_handlers[data_type].read() or '[]')
            
            # Append new item
            data.append(dict(adapter))
            
            # Write updated data
            self.file_handlers[data_type].seek(0)
            self.file_handlers[data_type].truncate()
            json.dump(data, self.file_handlers[data_type], ensure_ascii=False, indent=2)
            
        return item
    
    def update_visualization(self):
        """Update the HTML visualization with the latest data"""
        try:
            # Load visualization module dynamically
            vis_path = os.path.join(os.path.dirname(os.path.dirname(__file__)), 'visualization.py')
            
            if os.path.exists(vis_path):
                # Import the module
                spec = importlib.util.spec_from_file_location("visualization", vis_path)
                vis_module = importlib.util.module_from_spec(spec)
                spec.loader.exec_module(vis_module)
                
                # Generate visualization
                visualizer = vis_module.FinanceDataVisualizer(data_dir=self.data_dir)
                visualizer.generate_visualization()
                print("Visualization updated successfully.")
            else:
                print(f"Visualization module not found at {vis_path}")
        except Exception as e:
            print(f"Error updating visualization: {e}")
